import requests
from lxml import etree
import csv
import os

def writerRow(rowData):
    with open("./cityData.csv", "a", encoding="UTF-8", newline="") as csvfile:
        writer = csv.writer(csvfile)
        writer.writerow(rowData)

def init():
    if not os.path.exists('./cityData.csv'):
        with open("./cityData.csv", "w", encoding="UTF-8", newline="") as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow([
                'city',
                'cityLink'
            ])

def get_html(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
    }
    response = requests.get(url,headers=headers)
    if response.status_code == 200:
        return response.text
    else:
        return None

def parse_html(html):
    root = etree.HTML(html)
    cityList = root.xpath('//div[@class="fc-main clear"]//li//a')
    for city in cityList:
        cityName = city.text
        cityLink = city.get('href') + '/loupan/pg1/?_t=1'
        writerRow([cityName,cityLink])


def main():
    url = "http://bj.fang.lianjia.com/loupan/"  # 设置要爬取的网页 URL
    init()
    html = get_html(url)  # 发起请求获取网页 HTML
    if html:
        data = parse_html(html)  # 解析HTML获取数据

if __name__ == "__main__":
    main()
